#!/bin/bash

########
## filename:Distcp.sh
## use distcp transport passlog
## hongxing.fan
########

dir=$(dirname $0)
cd $dir

source ~/.bashrc
source /etc/profile

## which tpl appeared
HADOOP_BIN=/home/work/hadoop-client/hadoop/bin
SRC_PATH=/app/passport/pass_data/pass_data.db

REMOTE_PATH=hdfs://szwg-stoff-hdfs.dmop.baidu.com:54310/app/passport/pass_data/pass_data.db
LOCAL_PATH_59=webhdfs://nj03-game-m22dianquan59.nj03.baidu.com:8570/user/hive/warehouse/pass_data.db
LOCAL_PATH_58=webhdfs://nj03-game-m22dianquan58.nj03.baidu.com:8570/user/hive/warehouse/pass_data.db
LOCAL_PATH=$LOCAL_PATH_59


function isExist {
	if [[ $# -eq 1 ]]
	then
		tab_name=$1
		num=$($HADOOP_BIN/hadoop dfs -ls $SRC_PATH/$tab_name/dt=$cur_date | sort | uniq | wc -l)
		echo $num
		#if [[ $num -gt 1 ]];then
		#	echo 0
		#else
		#	echo 1
		#fi
	elif [[ $# -eq 2 ]]
	then
		tab_name=$1
		tpl=$2
		num=$($HADOOP_BIN/hadoop dfs -ls $SRC_PATH/$tab_name/tpl=$tpl/dt=$cur_date | awk -F" " '{print $8}' | awk -F"tpl=|/dt=" '{print $2}' | sort | uniq | wc -l)
		echo $num
		#if [[ $num -gt 100 ]];then
		#	echo 0
		#else
		#	echo 1
		#fi
	else
		echo "usage:sh isExist tablename [tpl]"
		exit 1
	fi
}

function transport {
	if [[ $# -eq 1 ]]
	then
		tab_name=$1
		echo "****$REMOTE_PATH/$tab_name/dt=$cur_date<==>$LOCAL_PATH/$tab_name/dt=$cur_date*****"
		{
			tryNum=0
			$HADOOP_BIN/hadoop distcp -update -i -m 800 $REMOTE_PATH/$tab_name/dt=$cur_date $LOCAL_PATH/$tab_name/dt=$cur_date
			stat=$?
			## 控制10个进程
			while [[ $stat -ne 0 && $tryNum -le 10 ]]
			do
				## 切换namenode
				tryNum=$(expr $tryNum + 1)
				if [[ `expr $tryNum % 2` -eq 0 ]];then
					LOCAL_PATH=$LOCAL_PATH_59
				else
					LOCAL_PATH=$LOCAL_PATH_58
				fi
				$HADOOP_BIN/hadoop distcp -update -i -m 800 $REMOTE_PATH/$tab_name/dt=$cur_date $LOCAL_PATH/$tab_name/dt=$cur_date
				stat=$?
			done
		}&
	elif [[ $# -eq 2 ]]
	then
		tab_name=$1
		tpl=$2
		tpls=$($HADOOP_BIN/hadoop dfs -ls $SRC_PATH/$tab_name/tpl=$tpl/dt=$cur_date | awk -F" " '{print $8}' | awk -F"tpl=|/dt=" '{print $2}' | sort | uniq)
		for t in ${tpls[*]}
		do
			num=$(ps aux | grep /home/work/hadoop-client/hadoop/bin/ | wc -l | cut -f1)
			num=$(expr $num - 1)
			while [[ $num -gt 20 ]]
			do
				sleep 10
				num=$(ps aux | grep /home/work/hadoop-client/hadoop/bin/ | wc -l | cut -f1)
				num=$(expr $num - 1)
			done
			echo "****$REMOTE_PATH/tpl=$t/dt=$cur_date<==>$LOCAL_PATH/tpl=$t/dt=$cur_date*****"
			{
				tryNum=0
				$HADOOP_BIN/hadoop distcp -update -i -m 800 $REMOTE_PATH/$tab_name/tpl=$t/dt=$cur_date $LOCAL_PATH/$tab_name/tpl=$t/dt=$cur_date
				stat=$?
				## 控制10个进程
				while [[ $stat -ne 0 && $tryNum -le 10 ]]
				do
					## 切换namenode
					tryNum=$(expr $tryNum + 1)
					if [[ `expr $tryNum % 2` -eq 0 ]];then
						LOCAL_PATH=$LOCAL_PATH_59
					else
						LOCAL_PATH=$LOCAL_PATH_58
					fi
					$HADOOP_BIN/hadoop distcp -update -i -m 800 $REMOTE_PATH/$tab_name/tpl=$t/dt=$cur_date $LOCAL_PATH/$tab_name/tpl=$t/dt=$cur_date
					stat=$?
				done
				echo "****$(date +'%Y-%m-%d %H:%M:%S')<==>path=$t/dt=$cur_date<==>status=$stat<==>tryNum=$tryNum****"
			}&
		done
		wait
	else
		echo "usage:sh transport tablename [tpl]"
		exit 1
	fi
}

cur_date=$(date +"%Y%m%d" -d "-1 day")
while getopts 'd:l:r:s:t:' OPT; do
	case $OPT in
		d)
			cur_date=$OPTARG
			;;
		l)
			flag=`isExist "$OPTARG"`
			while [[ $flag -eq 0 ]];do
				sleep $(expr 60 \* 10)
				flag=`isExist "$OPTARG"`
			done
			transport "$OPTARG"
			;;
		r)
			flag=`isExist "$OPTARG"`
			while [[ $flag -eq 0 ]];do
				sleep $(expr 60 \* 10)
				flag=`isExist "$OPTARG"`
			done
			transport "$OPTARG"
			;;
		s)
			flag=`isExist "$OPTARG" "*"`
			while [[ $flag -eq 0 ]];do
				sleep $(expr 60 \* 10)
				flag=`isExist "$OPTARG" "*"`
			done
			transport "$OPTARG" "*"
			;;
		t)
			flag=`isExist "ssn_log" "$OPTARG"`
			while [[ $flag -eq 0 ]];do
				sleep $(expr 60 \* 10)
				flag=`isExist "ssn_log" "$OPTARG"`
			done
			transport "ssn_log" "$OPTARG"
			;;
		*)
			echo "sh Distcp.sh -d 20150501 -l login_log -r reg_log -s ssn_log -t tb"
			;;
	esac
done

phone='13426082139 13811063282 13261676918 13520011291'
phone='13811063282'
host=$(hostname)
function sendMsg(){
	for number in $phone
	do
		gsmsend-script $number@$host pass_data.$1 "dt=$cur_date" "local=$2" "remote=$3"
	done
}

function compareNum(){
	if [[ $1 == "login_log" || $1 == "reg_log" ]];then
		r_num=$($HADOOP_BIN/hadoop dfs -ls $REMOTE_PATH/$1/dt=$cur_date/* | wc -l)
		files=$($HADOOP_BIN/hadoop dfs -ls $LOCAL_PATH_59/$1/*$cur_date/*)
		stat=$?
		if [[ $stat -ne 0 ]];then
			l_num=$($HADOOP_BIN/hadoop dfs -ls $LOCAL_PATH_58/$1/*$cur_date/* | wc -l)
		else
			l_num=$($HADOOP_BIN/hadoop dfs -ls $LOCAL_PATH_59/$1/*$cur_date/* | wc -l)
		fi
		if [[ $r_num -ne $l_num ]];then
			sendMsg $1 $l_num $r_num
		fi
	elif [[ $1 == "ssn_log" ]];then
		r_num=$($HADOOP_BIN/hadoop dfs -ls $REMOTE_PATH/$1/*/dt=$cur_date/* | wc -l)
		files=$($HADOOP_BIN/hadoop dfs -ls $LOCAL_PATH_59/$1/*/*$cur_date/*)
		stat=$?
		if [[ $stat -ne 0 ]];then
			echo "-----"
			l_num=$($HADOOP_BIN/hadoop dfs -ls $LOCAL_PATH_58/$1/*/*$cur_date/* | wc -l)
		else
			l_num=$($HADOOP_BIN/hadoop dfs -ls $LOCAL_PATH_59/$1/*/*$cur_date/* | wc -l)
		fi
		if [[ $r_num -ne $l_num ]];then
			sendMsg $1 $l_num $r_num
		fi
	fi
}
compareNum "login_log"
compareNum "reg_log"
compareNum "ssn_log"
